#!/bin/bash

set +e

iplast=$4
if [ -z "$4"   ]
then
  # 变量引用是$()，而不是${}
  hostname=$(hostname) 
  iplast=${hostname:7}
fi

num=$(($3-1))
ipport=$(($3+2345))
IPPrefix="192.168.8."

if(($3==5 || $3==6 || $3==7 || $3==8)); then
  num=$(($3-5))
fi

unseal=false
addpiece=false
precommit1=false
precommit2=false
commit=true
FetchLimit=2

ulimit -n 1048576
ulimit -Hn 1048576
ulimit -SHn 104857

if [ -z "$3"   ]
then
  echo "请输入序号: 1, 2, 3, 4"
  exit 1;
fi

if((
  $3!=1 && $3!=2 && $3!=3 && $3!=4 &&
  $3!=5 && $3!=6 && $3!=7 && $3!=8
  )); then
  echo "启动序号错误..."
  exit 1;
fi

if [ -z "$5"   ]
then
  echo "请设置组别"
  exit 1;
fi
# source /home/ceshi/.bashrc

export RUSTFLAGS="-C target-cpu=native -g"
# ### 启用源码编译底层库：
export FFI_BUILD_FROM_SOURCE=1

## 需验证 跑GPU时这个设置成0是否全部使用GPU   0.875
# export BELLMAN_CPU_UTILIZATION=0     #0.05

# ### 开启 Rust 日志：
export RUST_LOG=debug
export RUST_BACKTRACE=full

# export BELLMAN_NO_GPU=true
### 最大化内存参数： FIL_PROOFS_MAXIMIZE_CACHING
export FIL_PROOFS_MAXIMIZE_CACHING=1
## 新版本功能
# export FIL_PROOFS_USE_MULTICORE_SDR=1
### 使用 GPU 进行 Precommit2 加速： FIL_PROOFS_USE_GPU_COLUMN_BUILDER
export FIL_PROOFS_USE_GPU_COLUMN_BUILDER=1
#使用GPU，构造Merkle树 when having GPU
export FIL_PROOFS_USE_GPU_TREE_BUILDER=1   

# BELLMAN_CUSTOM_GPU=GeForce RTX 2080 Ti:4352
# FIL_PROOFS_MAXIMIZE_CACHING=1
# FIL_PROOFS_USE_GPU_COLUMN_BUILDER=1
# FIL_PROOFS_USE_GPU_TREE_BUILDER=1
# FIL_PROOFS_USE_MULTICORE_SDR=1


# BELLMAN_CUSTOM_GPU="<NAME>:<NUMBER_OF_CORES>"
# export BELLMAN_CUSTOM_GPU="GeForce RTX 2080 Ti"
# export BELLMAN_CUSTOM_GPU="GeForce GTX 1080:2560"
# BELLMAN_CUSTOM_GPU="GeForce RTX 2070 SUPER:2560"

# CUDA_VISIBLE_DEVICES=1     # 只有编号为1的GPU对程序是可见的，在代码中gpu[0]指的就是这块儿GPU
# CUDA_VISIBLE_DEVICES=0,2,3 # 只有编号为0,2,3的GPU对程序是可见的，在代码中gpu[0]指的是第0块儿，
# 						   # gpu[1]指的是第2块儿，gpu[2]指的是第3块儿
# CUDA_VISIBLE_DEVICES=2,0,3 # 只有编号为0,2,3的GPU对程序是可见的，但是在代码中gpu[0]指的是第2块儿，
# 						   # gpu[1]指的是第0块儿，gpu[2]指的是第3块儿
export GPU_DEVICE_ORDINAL=$num      #0,1,2,3  $num
export CUDA_VISIBLE_DEVICES=$num      #0,1,2,3  $num

# P2 环节
# FIL_PROOFS_USE_GPU_COLUMN_BUILDER - 使用GPU，进行column hash的计算
# FIL_PROOFS_USE_GPU_TREE_BUILDER - 使用GPU，构造Merkle树


# Chunk-size: 7488999 默认
# Chunk-size: 44785859
# # FIL_PROOFS_MAX_GPU_COLUMN_BATCH_SIZE - 每次计算Column的batch大小，默认400000
# export FIL_PROOFS_MAX_GPU_COLUMN_BATCH_SIZE=1200000
# # FIL_PROOFS_COLUMN_WRITE_BATCH_SIZE - 每次刷Column数据的batch大小，默认262144
# export FIL_PROOFS_COLUMN_WRITE_BATCH_SIZE=2097152
# # FIL_PROOFS_MAX_GPU_TREE_BATCH_SIZE - 每次Encoding计算的batch大小，默认700000
# export FIL_PROOFS_MAX_GPU_TREE_BATCH_SIZE=2100000


# FIL_PROOFS_MAXIMIZE_CACHING

# FIL_PROOFS_USE_GPU_TREE_BUILDER - 使用GPU，构造Merkle树

# FIL_PROOFS_MAX_GPU_TREE_BATCH_SIZE - 每次Encoding计算的batch大小，默认700000

# WORKER="/var/tmp"
WORKER="/home/ceshi/.lotusworker"

### 临时文件夹路径： TMPDIR
export TMPDIR=$WORKER/tmp_c2_$3
### PreCommit1 proof parents 缓存路径（可以减少 56G 内存）
export FIL_PROOFS_PARENT_CACHE="$WORKER/filecoin-parents"
### proof 证明参数路径： FIL_PROOFS_PARAMETER_CACHE
export FIL_PROOFS_PARAMETER_CACHE="$WORKER/filecoin-proof-parameters"

echo "FULLNODE_API_INFO: " $1
echo "MINER_API_INFO: " $2

# export $1
export $2


# export LOTUS_WORKER_PATH=/home/ceshi/.worker/.worker3
# OPTIONS:
#    --listen value                工作者api将侦听的主机地址和端口 (default: "0.0.0.0:3456")
#    --no-local-storage            不将storageminer存储库用于扇区存储 (default: false)
#    --no-swap                     不使用交换 (default: false)
#    --addpiece                    启用addpiece (default: true)
#    --precommit1                  enable precommit1 (32G sectors: 1 core, 128GiB Memory) (default: true)
#    --unseal                      启用开封 (32G sectors: 1 core, 128GiB Memory) (default: true)
#    --precommit2                  enable precommit2 (32G sectors: all cores, 96GiB Memory) (default: true)
#    --commit                      enable commit (32G sectors: all cores or GPUs, 128GiB Memory + 64GiB swap) (default: true)
#    --parallel-fetch-limit value  可并行运行的最大读取操作 (default: 5)
#    --timeout value               当未指定“ listen”时使用。必须是golang的时间可以识别的有效持续时间。ParseDuration函数 (default: "30m")
#    --help, -h                    show help (default: false)
# export LOTUS_WORKER_ARGS="--no-local-storage=false --parallel-fetch-limit=1 --addpiece=false --unseal=false --precommit1=false --precommit2=false --commit=true --listen=10.0.1.234:2348"
# export LOTUS_WORKER_LOG=/home/ceshi/all78.log


if [ ! -d $TMPDIR ]; then
  mkdir $TMPDIR
  echo "mkdir ${TMPDIR} done"
fi

# 清理
rm $TMPDIR/*
echo "清除 ${TMPDIR} 目录文件"

echo "start worker$3 to $IPPrefix$iplast:$ipport...  GPU启动编号: $num"

# WORKER="/home/ceshi/.lotusworker"
WORKERREPO="${WORKER}/.worker$3"

ps -fe|grep ${WORKERREPO} |grep -v grep
if [ $? -ne 0 ]
then

  echo "start process....."

  nohup /usr/local/bin/lotus-worker --worker-repo=${WORKERREPO} run  --no-local-storage=false --addpiece=$addpiece --unseal=$unseal --precommit1=$precommit1 --precommit2=$precommit2 --commit=$commit   --group=$5 --ability="AP:1,PC1:1,PC2:1,C1:1,C2:1"  --listen=$IPPrefix$iplast:$ipport --parallel-fetch-limit=$FetchLimit >> ~/allc2$3.log 2>&1 &

  
  if [ -z "$4"   ]
  then
    ps -ef | grep lotus-worker
  fi
else
  echo "runing....."
fi



# nohup /usr/local/bin/lotus-worker --worker-repo=~/.worker/.worker$3 run >> all$3.log 2>&1 &

# 

# export CUDA_VISIBLE_DEVICES=2
# 
# 
# export LOTUS_WORKER_LOG=worker-c-gpu$CUDA_VISIBLE_DEVICES
# export TMPDIR=/cache/tmp/gpu$CUDA_VISIBLE_DEVICES

# watch -n 1 nvidia-smi
# nvidia-smi -q

# NUMA node0 CPU(s):               0-15,32-47
# NUMA node1 CPU(s):               16-31,48-63

# ps -eo pid,args,psr | grep lotus-worker

# taskset -cp 0-15,32-47 64121
# taskset -cp 16-31,48-63 64663